{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sliding Window Attention is enabled but not implemented for `sdpa`; unexpected results may be encountered.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer, AutoProcessor\n",
    "clip_model_dir = \"./download/clip-vit-large-patch14-336\" # huggingface上先下载好\n",
    "qwen_model_dir = \"./download/Qwen2.5-1.5B-Instruct\" # huggingface上先下载好\n",
    "clip_model = AutoModel.from_pretrained(clip_model_dir, device_map=\"cuda:0\")\n",
    "qwen_model = AutoModelForCausalLM.from_pretrained(qwen_model_dir, torch_dtype=\"auto\", device_map=\"cuda:0\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(qwen_model_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sliding Window Attention is enabled but not implemented for `eager`; unexpected results may be encountered.\n"
     ]
    }
   ],
   "source": [
    "from transformers import (\n",
    "    LlavaForConditionalGeneration,\n",
    "    LlavaConfig\n",
    ")\n",
    "vision_config = clip_model.vision_model.config\n",
    "text_config = qwen_model.config\n",
    "configuration = LlavaConfig(vision_config, text_config)\n",
    "model = LlavaForConditionalGeneration(configuration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CLIPVisionConfig {\n",
       "  \"_attn_implementation_autoset\": true,\n",
       "  \"attention_dropout\": 0.0,\n",
       "  \"dropout\": 0.0,\n",
       "  \"hidden_act\": \"quick_gelu\",\n",
       "  \"hidden_size\": 1024,\n",
       "  \"image_size\": 336,\n",
       "  \"initializer_factor\": 1.0,\n",
       "  \"initializer_range\": 0.02,\n",
       "  \"intermediate_size\": 4096,\n",
       "  \"layer_norm_eps\": 1e-05,\n",
       "  \"model_type\": \"clip_vision_model\",\n",
       "  \"num_attention_heads\": 16,\n",
       "  \"num_channels\": 3,\n",
       "  \"num_hidden_layers\": 24,\n",
       "  \"patch_size\": 14,\n",
       "  \"projection_dim\": 768,\n",
       "  \"torch_dtype\": \"float32\",\n",
       "  \"transformers_version\": \"4.49.0\"\n",
       "}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vision_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.vision_tower.vision_model = clip_model.vision_model\n",
    "model.language_model = qwen_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151665"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.config.pad_token_id = tokenizer.pad_token_id\n",
    "model.config.image_token_index = tokenizer.encode(\"<image>\")[0]\n",
    "model.config.image_token_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "autoprocessor = AutoProcessor.from_pretrained(clip_model_dir)\n",
    "autoprocessor.save_pretrained(\"show_model/model002\") #注意这里是model002"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer.save_pretrained(\"show_model/model001\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"show_model/model001\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 重点\n",
    "在show_model/model002/preprocessor_config.json文件中增加一个key value对<\"patch_size\": 14>，并将文件拷贝一份到show_model/model001目录下。  \n",
    "如果不加<\"patch_size\": 14>这个键值对，后面就没办法给llava_processor.patch_size赋值（看后面代码）\n",
    "### 做完上面的步骤重启jupyter，测试一下效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from transformers import LlavaProcessor, LlavaForConditionalGeneration, AutoProcessor\n",
    "import torch\n",
    "\n",
    "\n",
    "model_name_or_path = \"show_model/model001\"  # \n",
    "\n",
    "llava_processor = LlavaProcessor.from_pretrained(model_name_or_path)\n",
    "tokenizer = llava_processor.tokenizer\n",
    "#这里必须要给llava_processor.patch_size赋值，否则llava_processor.patch_size==None，后面会报错。注意preprocessor_config.json需要有<\"patch_size\": 14>这个键值对才行\n",
    "llava_processor.patch_size = llava_processor.image_processor.patch_size\n",
    "\n",
    "model = LlavaForConditionalGeneration.from_pretrained(\n",
    "    model_name_or_path, device_map=\"cuda:0\", torch_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "prompt_text = \"<image>\\nWhat are these?\"\n",
    "messages = [\n",
    "    {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
    "    {\"role\": \"user\", \"content\": prompt_text},\n",
    "]\n",
    "prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
    "\n",
    "image_path = \"./000000039769.jpg\"\n",
    "image = Image.open(image_path)\n",
    "inputs = llava_processor(text=prompt, images=image, return_tensors=\"pt\") #这里调用的是llava_processor.__call__()，用到了llava_processor.patch_size，因此必须保证llava_processor.patch_size非空\n",
    "\n",
    "for tk in inputs.keys():\n",
    "    inputs[tk] = inputs[tk].to(model.device)\n",
    "\n",
    "generate_ids = model.generate(**inputs, max_new_tokens=20)\n",
    "print(tokenizer.decode(list(generate_ids[0])))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test_0121",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
